package net.gdface.service.client;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import net.gdface.db.DatabaseManager;
import net.facelib.u4fdb.localdb.TaskBean;
import net.gdface.facedb.db.ImageBean;
import net.gdface.facedb.DuplicateRecordException;
import net.gdface.image.ImageErrorException;
import net.gdface.sdk.NotFaceDetectedException;
import net.gdface.service.search.SaveWorker;
import net.gdface.utils.Assert;
import net.gdface.utils.BinaryUtils;
import net.gdface.utils.SimpleTypes;
import net.gdface.utils.ImageUtlity;
import net.gdface.utils.PathUtility;
import net.gdface.worker.QueueManager;
import net.gdface.worker.QueueManagerImpl;
import net.gdface.worker.WorkData;
import net.gdface.worker.WorkerManager;
import net.gdface.worker.WorkerManagerFactory;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;

import static net.gdface.db.DaoManagement.DAO;
import static com.google.common.base.Preconditions.*;

/**
 * 扫描本地文件文件夹的图像文件,将图像文件加入FaceDB数据库<br>
 * @author guyadong
 *
 */
public  class AddImages extends LocalApp implements SaveImage{
	private static final Logger logger = LoggerFactory.getLogger(AddImages.class);
	protected static final String APP_NAME = "ADD IMAGE";
	///////////////////////配置变量,在loadParametersFromProperties中初始化///////////////////
	/**
	 * 删除本地重复文件标志
	 */
	protected boolean cleanDuplication ;
	/**
	 * 删除数据库中重复文件标志
	 */
	protected boolean cleanDupInDB ;
	/**
	 * 删除非图像文件标志
	 */
	protected boolean cleanNoImage;
	/**
	 * 删除无人脸图像文件标志
	 */
	protected boolean cleanNoface ;
	private Thread threadQueueProducer;

	protected AtomicLong total_faces = new AtomicLong(0);
	protected AtomicLong total_images = new AtomicLong(0);
	private long startMills = System.currentTimeMillis();
	//private BlockingQueue<TaskBean> queue;
	private QueueManager<WorkDataFile> queueLevel2;
	private WorkerManager workerPoolLevel2;
	
	protected AddImages()  {
		super();
		
		options.addOption(Option.builder(CLEAN_DUPLICATION_OPTION).longOpt(CLEAN_DUPLICATION_OPTION_LONG)
				.desc(CLEAN_DUPLICATION_OPTION_DESC).build());
		options.addOption(Option.builder(CLEAN_NOFACE_OPTION).longOpt(CLEAN_NOFACE_OPTION_LONG)
				.desc(CLEAN_NOFACE_OPTION_DESC).build());
		options.addOption(Option.builder(CLEAN_NOIMAGE_OPTION).longOpt(CLEAN_NOIMAGE_OPTION_LONG)
				.desc(CLEAN_NOIMAGE_OPTION_DESC).build());
		options.addOption(Option.builder(CLEAN_DUPINDB_OPTION).longOpt(CLEAN_DUPINDB_OPTION_LONG)
				.desc(CLEAN_DUPINDB_OPTION_DESC).build());
	}

	@Override
	public void loadConfig(Options options, CommandLine cmd) throws ParseException {
		super.loadConfig(options, cmd);
		if(cmd.hasOption(CLEAN_DUPLICATION_OPTION_LONG)){
			cleanDuplication  = true;
		}
		if(cmd.hasOption(CLEAN_NOFACE_OPTION_LONG)){
			cleanNoface  = true;
		}
		if(cmd.hasOption(CLEAN_NOIMAGE_OPTION_LONG)){
			cleanNoImage  = true;
		}
		if(cmd.hasOption(CLEAN_DUPINDB_OPTION_LONG)){
			cleanDupInDB  = true;
		}
	}

	@Override
	protected void loadParametersFromProperties() {
		super.loadParametersFromProperties();
		CONFIG.setPrefix(AddImages.class.getName()+".");
		this.cleanDuplication = CONFIG.getPropertyBaseType("cleanDuplications", false);
		this.cleanDupInDB = CONFIG.getPropertyBaseType("cleanDupInDB", false);
		this.cleanNoImage = CONFIG.getPropertyBaseType("cleanNoImage", false);
		this.cleanNoface = CONFIG.getPropertyBaseType("cleanNoface", false);
	}
	
	/**
	 * 删除bean指定的文件
	 * @param bean
	 * @param cause
	 */
	protected void removeFile(TaskBean bean,Status cause){
		new File(this.getAbsolutePath(bean.getFile())).delete();
		logger.info("{} DELETED {}", cause.getMsg(), bean.getFile());
	}
	
	/**
	 * 保存任务对象<br>
	 * 根据任务状态({@link TaskBean#getStatus()})和clean参数( {@link #cleanDupInDB},{@link #cleanNoface},{@link #cleanNoImage})决定是保存记录到本地数据库还是删除添加失败的文件
	 * @param bean
	 */
	protected void _saveTask(TaskBean bean) {
		Status cause = null;
		Status status = Status.valueOf(bean.getStatus());
		if (status == Status.NOT_FACE && this.cleanNoface)cause = status;
		else if (status == Status.DUPINDB && this.cleanDupInDB)cause = status;
		else if (status == Status.NOTIMAGE && this.cleanNoImage)	cause = status;
		else if (status == Status.ZERODATA && this.cleanNoImage)	cause = status;
		else if (status == Status.NOTFOUND && this.cleanNoImage)	cause = status;

		if (cause != null) {
			removeFile(bean, cause);
			DAO.daoDeleteTask(bean);
		}else{
			DAO.daoSaveTask(bean);
		}
	}

	@Override
	protected void startApp() throws InterruptedException {
		this.workerPoolLevel2.startManager();
		monitorThreadStart();		
		try {
			//等待生产线程结束
			produceThreadStart().join();
		} finally {
			this.workerPoolLevel2.stopManager(300, TimeUnit.SECONDS);
		}
	}
	
	@Override
	protected void onFinished() {
		super.onFinished();		
		double timeSeconds = (double) TimeUnit.SECONDS.convert(System.currentTimeMillis() - this.startMills,
				TimeUnit.MILLISECONDS);
		logger.info("共处理{}张图片(images),生成{}个特征码(faces)", this.total_images.get(), this.total_faces.get());
		logger.info(String.format("用时%.2f senconds(秒),average(平均):%.2f images/s %.2f codes/s", timeSeconds,
				this.total_images.get() / timeSeconds, this.total_faces.get() / timeSeconds));
	}

	protected String getAbsolutePath(String f) {
		return PathUtility.getAbsolutePath(this.root, f);
	}

	protected String getRelativeDbPath(File f) {
		return PathUtility.getRelativeDbPath(this.root, f);
	}
	
	@Override
	protected void createDbInstance() throws Exception {
		DatabaseManager.createSingleton(root, reSearch, runDbInMemory);
	}
	
	@Override
	protected void init() throws Exception  {
		super.init();
		try{
			this.queueLevel2 = new QueueManagerImpl<WorkDataFile>(1*1000, "SQ", this.workQueueSize);
			//注册侦听器，处理SQLIntegrityConstraintViolationException异常
			DAO.taskDuplicationListener.register((bean)->{
					if(cleanDuplication) {
						logger.info("Delete duplicated file {}", bean.getFile());
						new File(getAbsolutePath(bean.getFile())).delete();
					}
				});
			SaveWorker.set(queueLevel2, this, LocalApp.CUSTOM_RETRY_STRATEGY, 2000);
			this.workerPool.shutdownNow();		
			this.workerPoolLevel2=WorkerManagerFactory.newFixedThreadPool(this.maxWorkerThreads, SaveWorker.class, queueLevel2, 5000, "L2");
		}
		finally{
		}
	}

	/**
	 * 从数据库搜索没有完成的文件，加入队列
	 * @throws InterruptedException
	 */
	private void loadTask() throws InterruptedException {
		logger.info("begin search database for uncompleted file...");
		DAO.daoForeachTask((bean)->{
			switch (Status.valueOf(bean.getStatus())) {
			case UNEXP:
			case FAIL_SAVEDB:
			case NEW:
				logger.debug("{}:{}",bean.getStatus(),getAbsolutePath(bean.getFile()));
				if (new File(getAbsolutePath(bean.getFile())).exists()) {
					sendTask(bean);
					return false;
				} else{
					// 从数据库中删除不存在的文件
					logger.warn("NOT EXISTS FILE {}",getAbsolutePath(bean.getFile()));
					return true;
				}
			case FOLDEROK:
			case FOLDERUN:
			default:
				return false;
			}
		}, false);
	}

	/**
	 * 启动队列生产线程
	 * 
	 * @return
	 */
	protected Thread produceThreadStart() {
		this.threadQueueProducer = new Thread(new Runnable() {
			public void run() {
				logger.info("producer线程启动。。。");
				produceQueue();
				logger.info("producer线程结束");
			}
		}, "Producer");
		
		this.threadQueueProducer.start();
		return this.threadQueueProducer;
	}

	/**
	 * 启动监视线程
	 * 
	 * @return
	 */
	protected Thread monitorThreadStart() {
		Thread t = new Thread(new Runnable() {
			public void run() {
				logger.info("monitor线程启动。。。");
				try {
					while (!isFinished.get()) {
						Thread.sleep(5 * 1000);
						System.out.printf("[%s]%d_F_%d_IMG\r", Thread.currentThread().getName(),total_faces.get(), total_images.get());
					}
				} catch (InterruptedException e) {
					logger.info("{} interrupted", Thread.currentThread().getName());
					Thread.currentThread().interrupt();
				}
				logger.info("monitor线程结束。。。");
			}
		}, "MN");
		t.setDaemon(true);
		t.start();
		return t;
	}

	protected void produceQueue() {
		try {
			logger.info("队列produce开始");
			loadTask();
			logger.info("Search folder for images...");
			searchImages(this.root);
			logger.info("队列produce结束");
		} catch (InterruptedException e) {
			logger.info("{} interrupted", Thread.currentThread().getName());
			Thread.currentThread().interrupt();
		}
	}

	/**
	 * 每一个文件处理完成后，更新其所在的工作的目录状态{@link #workFolders}，将未完成计数减1
	 * 
	 * @param bean
	 */
	private void _updateWorkFolder(TaskBean bean) {
		Status status = Status.valueOf(bean.getStatus());
		if (Status.SAVEDB_NORMAL_STATUS.contains(status)||status==Status.NOTFOUND||status==Status.ZERODATA){
			assert null!=bean.getFile()&&!bean.getFile().isEmpty();
			File f = new File(this.getAbsolutePath(bean.getFile()));
			assert f.isFile();
			dec(this.getRelativeDbPath(f.getParentFile()));				
		}
	}

	/**
	 * 搜索指定文件夹,将文件夹中的图像文件后缀的文件加入任务队列<br>
	 * 隐藏文件夹和derby数据库文件夹不在搜索范围
	 * 
	 * @param dir
	 *            文件夹路径,
	 * @throws InterruptedException
	 */
	protected void searchImages(File dir) throws InterruptedException {
		Assert.notNull(dir, "dir");
		// 检查结束标志
		if (this.isFinished.get())
			return;
		File[] list = dir.listFiles(new FileFilter() {
			// 检查文件后缀是否为支持的图片格式文件后缀
			public boolean accept(File pathname) {
				if (isFinished.get())
					return false;
				if (!pathname.isHidden()) {
					if (pathname.isFile()) {
						return ImageUtlity.isImageSuffix(pathname.getName())&&pathname.length()>0;
					} else if (pathname.isDirectory()) {
						// 避开搜索数据库目录
						if (pathname.getName().equals(DatabaseManager.TASK_DB)) {
							return false;
						} else {
							try {
								searchImages(pathname);
							} catch (InterruptedException e) {
							}
							return false;
						}
					}
				}
				return false;
			}
		});
		// 再次检查结束标志
		if (this.isFinished.get())
			return;
		// 将文件加入队列
		String pr = getRelativeDbPath(dir);
		logger.debug("Searching:{}", pr);
		assert null != list;
		if (list.length > 0 && !isFinishedFolder(pr)) {
			addNewIfAbsent(pr, list.length);
			for (File f : list) {
				if(Thread.interrupted())break;
				// 生成相对起始目录的相对路径
				String fr = getRelativeDbPath(f);
				// 找到记录就跳过
				TaskBean bean = DAO.daoGetTask(fr);
				// 数据库中找不到记录，就加入队列
				if (null == bean) {
					bean = TaskBean.builder().status(Status.NEW.name()).file(fr).build();
					sendTask(bean);
				} 
			}
		}
	}
	private static void addNewIfAbsent(String folder,int left) {
		checkArgument(left>=0," left >=  0 required");
		checkArgument(!Strings.isNullOrEmpty(folder),"folder is empty or null");
		Status status = 0 == left ? Status.FOLDEROK : Status.FOLDERUN;
		TaskBean bean = TaskBean.builder()
				.md5(BinaryUtils.getMD5String(folder.getBytes()))
				.file(folder)
				.value(left)
				.status(status.name())
				.build();
		DAO.daoAddTaskIfAbsent(bean);
	}
	
	private static int dec(String folder) {
		Assert.notEmpty(folder, "folder");
		TaskBean taskBean = DAO.daoGetTaskChecked(folder);
		Integer v = MoreObjects.firstNonNull(taskBean.getValue(), 0) - 1;
		taskBean.setValue(v);
		DAO.daoSaveTask(taskBean);
		return v.intValue();
	}
	
	private static boolean isFinishedFolder(String folder){
		Assert.notEmpty(folder, "folder");
		// 从数据库中获取当前文件夹的状态，如果不存在则为null
		TaskBean taskBean = DAO.daoGetTask(folder);
		Status folderStatus = null==taskBean?null:Status.valueOf(taskBean.getStatus());
		assert folderStatus == null || folderStatus == Status.FOLDEROK || folderStatus == Status.FOLDERUN;
		return folderStatus == Status.FOLDEROK;
	}
	private void sendTask(TaskBean bean) throws InterruptedException{
		File file = new File(this.getAbsolutePath(bean.getFile()));
		//使用绝对路径做构造方法的参数
		WorkDataFile woc = new WorkDataFile(file);
		woc.setVariables(WorkDataFile.VAR_TASKBEAN,bean);
		this.queueLevel2.push(woc);
		logger.debug("add image to queue {}", file.getAbsolutePath());
		
	}
	@Override
	protected String getAppName() {
		return APP_NAME;
	}
	
	protected TaskBean runFile(TaskBean bean) {
		File file = new File(this.getAbsolutePath(bean.getFile()));
		if(!file.exists()){
			bean.setStatus(Status.NOTFOUND.name());
			logger.info("{} {}", bean.getStatus(), file);
		}else	if (file.length() == 0) {
			bean.setStatus(Status.ZERODATA.name());
			logger.info("{} {}", bean.getStatus(), file);
		} else {
			String frp = bean.getFile();
			bean.setStatus(Status.FAIL_SAVEDB.name());
			try {
				ImageBean res = this.facedb.detectAndAddFeatures(file, addImageFaceNum);
				Integer taskValue = 0;
				bean.setMd5(res.getMd5());
				bean.setStatus(Status.SAVED_INDB.name());
				taskValue = res.getFaceNum();
				logger.debug("total[{}FACE/{}IMG]{} add ok,%d faces,file[{}]", total_faces.get(), total_images.get(),
						frp, res.getFaceNum(), file);
				this.total_images.incrementAndGet();

				bean.setValue(taskValue.intValue());
				this.total_faces.addAndGet(taskValue);
			} catch (DuplicateRecordException e) {
				onDuplicateRecordException(e,bean, file);
				logger.debug("{} {} {}:{}", bean.getStatus(), frp, e.getClass().getSimpleName(), e.getMessage());
			} catch (ImageErrorException e) {
				bean.setStatus(Status.ERRIMAGE.name());
				logger.warn("{} {} {}:{}", bean.getStatus(), frp, e.getClass().getSimpleName(), e.getMessage());
			} catch (IOException e) {
				bean.setStatus(Status.IOERROR.name());
				logger.warn("{} {} {}:{}", bean.getStatus(), frp, e.getClass().getSimpleName(), e.getMessage());
			} catch (NotFaceDetectedException e) {
				bean.setStatus(Status.NOT_FACE.name());
				logger.debug("{} {} {}:{}", bean.getStatus(), frp, e.getClass().getSimpleName(), e.getMessage());
			} catch (RuntimeException e) {
				// 忽略结束标志为true时抛出的所有异常
				if(!isFinished.get()){
					throw e;
				}
			}
		}
		return bean;
	}

	/**
	 * 处理{@link DuplicateRecordException}异常
	 * @param e
	 * @param bean
	 * @param file
	 */
	protected void onDuplicateRecordException(DuplicateRecordException e,TaskBean bean, File file){
		try {
			String md5 = BinaryUtils.getMD5String(BinaryUtils.getBytes(file));
			bean.setMd5(md5);
			bean.setStatus(Status.DUPINDB.name());
		} catch (IOException e1) {
			throw new RuntimeException(e);
		}
	}
	
	@Override
	public void save(WorkData woc) {
		TaskBean bean = woc.getVariables(WorkDataFile.VAR_TASKBEAN);
		assert null!=bean;
		runFile(bean);
	}

	@Override
	public void recordStatus(WorkData woc) {
		TaskBean bean = woc.getVariables(WorkDataFile.VAR_TASKBEAN);
		_updateWorkFolder(bean);
		_saveTask(bean);
	}

	@Override
	public boolean needSave(WorkData woc) {
		return true;
	}

	@Override
	public void onRuntimeException(RuntimeException e, WorkData woc) throws RuntimeException {
		Assert.notNull(e, "e");
		Assert.notNull(woc, "woc");
		TaskBean bean = woc.getVariables(WorkDataFile.VAR_TASKBEAN);
		Throwable fault = SimpleTypes.stripThrowableShell(e, RuntimeException.class);
		bean.setExp(fault.getClass().getName());
	}

	public static void main(String[] args) throws Exception {
		new AddImages()
			.parseCommandLine(args)
			.self(AddImages.class)
			.start();
	}

}